{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12 relations in dev dataset in total\n",
      "to-M: 1.0/12 (0.08333333333333333)\n",
      "to-1: 11.0/12 (0.9166666666666666)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "import sys\n",
    "\n",
    "\n",
    "def get_relations_by_type(data_dir):\n",
    "    with open(os.path.join(data_dir, 'raw.kb')) as f:\n",
    "        triples = list(f.readlines())\n",
    "    with open(os.path.join(data_dir, 'train.triples')) as f:\n",
    "        triples += list(f.readlines())\n",
    "    triples = list(set(triples))\n",
    "\n",
    "    query_answers = dict()\n",
    "\n",
    "    theta_1_to_M = 1.5\n",
    "\n",
    "    for triple_str in triples:\n",
    "        e1, e2, r = triple_str.strip().split('\\t')\n",
    "        if not r in query_answers:\n",
    "            query_answers[r] = dict()\n",
    "        if not e1 in query_answers[r]:\n",
    "            query_answers[r][e1] = set()\n",
    "        query_answers[r][e1].add(e2)\n",
    "\n",
    "    to_M_rels = set()\n",
    "    to_1_rels = set()\n",
    "\n",
    "    dev_rels = set()\n",
    "    with open(os.path.join(data_dir, 'dev.triples')) as f:\n",
    "        for line in f:\n",
    "            e1, e2, r = line.strip().split('\\t')\n",
    "            dev_rels.add(r)\n",
    "\n",
    "    num_rels = len(dev_rels)\n",
    "    print('{} relations in dev dataset in total'.format(num_rels))\n",
    "    for r in dev_rels:\n",
    "        ratio = np.mean([len(x) for x in query_answers[r].values()])\n",
    "        if ratio > theta_1_to_M:\n",
    "            to_M_rels.add(r)\n",
    "        else:\n",
    "            to_1_rels.add(r)\n",
    "\n",
    "    num_to_M = len(to_M_rels) + 0.0\n",
    "    num_to_1 = len(to_1_rels) + 0.0\n",
    "    print('to-M: {}/{} ({})'.format(num_to_M, num_rels, num_to_M / num_rels))\n",
    "    print('to-1: {}/{} ({})'.format(num_to_1, num_rels, num_to_1 / num_rels))\n",
    "    return to_M_rels, to_1_rels\n",
    "\n",
    "    \n",
    "def main():\n",
    "    dataset = 'NELL-995'\n",
    "    data_dir = os.path.join('/home/xilin/Projects/dtt/data', dataset)\n",
    "    group_relation_by_type(data_dir)\n",
    "    \n",
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "154213 unique facts\n"
     ]
    }
   ],
   "source": [
    "\"\"\"Count number of unique facts in each KG\"\"\"\n",
    "\n",
    "\n",
    "# with open(os.path.join(data_dir, 'dev.triples')) as f:\n",
    "#     triples += list(f.readlines())\n",
    "# with open(os.path.join(data_dir, 'test.triples')) as f:\n",
    "#     triples += list(f.readlines())\n",
    "    \n",
    "print('{} unique facts'.format(len(set(triples))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
